package com.simoniu.sparkdemo.javademo;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
public class CreateRddByFileJavaDemo {
    public static void main(String[] args) {
        //创建SparkContext：
        SparkConf conf = new SparkConf();
        conf.setAppName("CreateRddByFileJava")
                .setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);

        String path = "D:\\uploadFiles\\hello2.txt";
        //path = "hdfs://bigdata01:9000/test/hello.txt";
        //读取文件数据，可以在textFile中指定生成的RDD的分区数量
        JavaRDD<String> rdd = sc.textFile(path,2);
        //获取每一行数据的长度
        JavaRDD<Integer> lengthRDD = rdd.map(new Function<String, Integer>() {
            @Override
            public Integer call(String line) throws Exception {
                return line.length();
            }
        });
        //计算文件内数据的总长度
        Integer length = lengthRDD.reduce(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });
        System.out.println("length="+length);
        sc.stop();
    }
}
